
mkdir dump
mkdir dump.post
mkdir gen
mkdir peruser

if [ ! -f dic/dict.opcorpora.txt ]; then 
  cd dic/
  bzip2 -d -k dict.opcorpora.txt.bz2
  cd ..;
fi

perl scripts/remove_tags.pl < $1 

mv fulltext.xml gen/

export LC_ALL="C"

# wordforms
perl scripts/export_ngram.pl -f gen/fulltext.xml -i -c -l -n 1 | sort > gen/fulltext.n1
perl scripts/export_ngram.pl -f gen/fulltext.xml -i -c -l -n 2 | sort > gen/fulltext.n2

# lemmata
perl scripts/export_ngram.pl -f gen/fulltext.xml -i -c -l -n 1 -L | sort > gen/fulltext.l1
perl scripts/export_ngram.pl -f gen/fulltext.xml -i -c -l -n 2 -L | sort > gen/fulltext.l2

# wordforms
perl scripts/colloc.pl -m MI -u gen/fulltext.n1 -b gen/fulltext.n2 -t > gen/fulltext.MI
perl scripts/colloc.pl -m T-score -u gen/fulltext.n1 -b gen/fulltext.n2 > gen/fulltext.TS

# lemmata
perl scripts/colloc.pl -m MI -u gen/fulltext.l1 -b gen/fulltext.l2 -t > gen/fulltext.lMI
perl scripts/colloc.pl -m T-score -u gen/fulltext.l1 -b gen/fulltext.l2 > gen/fulltext.lTS

#exit

perl scripts/unigen.pl -f gen/fulltext.xml | sort > gen/fulltext.uni
perl scripts/unigen.pl -f gen/fulltext.xml -b gen/fulltext.MI | sort > gen/fulltext.uni2

perl scripts/unigen.pl -f gen/fulltext.xml -l | sort > gen/fulltext.luni
perl scripts/unigen.pl -f gen/fulltext.xml -b gen/fulltext.MI -l | sort > gen/fulltext.luni2

for f in `ls peruser/*.xml | grep -E -o "[0-9]+"`; 
do
  perl scripts/unigen.pl -f peruser/user$f.xml | sort > peruser/user$f.uni
  perl scripts/unigen.pl -f peruser/user$f.xml -b gen/fulltext.MI | sort > peruser/user$f.uni2

  perl scripts/unigen.pl -f peruser/user$f.xml -l | sort > peruser/user$f.luni
  perl scripts/unigen.pl -f peruser/user$f.xml -b gen/fulltext.MI -l | sort > peruser/user$f.luni2
done

for f in `ls peruser/*.xml | grep -E -o "[0-9]+"`; do gawk '{print $1}' peruser/user$f.uni2 | grep -v '^###TOTAL###'; done | sort | uniq -c | sort -r -n | gawk '{print $2"\t"$1}' > gen/word2nuser.txt

for f in `ls peruser/*.xml | grep -E -o "[0-9]+"`; do gawk '{print $1}' peruser/user$f.luni2 | grep -v '^###TOTAL###'; done | sort | uniq -c | sort -r -n | gawk '{print $2"\t"$1}' > gen/word2nuser.ltxt

totalusers=$( ls peruser/*.xml | wc -l )

for f in `ls peruser/*.xml | grep -E -o "[0-9]+"`; do echo $f $( grep -c "<text" peruser/user$f.xml ); done > gen/user2ndocs.txt

echo $totalusers

for f in `ls peruser/*.xml | grep -E -o "[0-9]+"`; 
do
  echo
  echo $f
  echo

  join peruser/user$f.uni gen/fulltext.uni > peruser/user$f.join
  join peruser/user$f.uni2 gen/fulltext.uni2 > peruser/user$f.join2

  join peruser/user$f.luni gen/fulltext.luni > peruser/user$f.ljoin
  join peruser/user$f.luni2 gen/fulltext.luni2 > peruser/user$f.ljoin2

  # use wordforms
  perl scripts/calc_mi.pl < peruser/user$f.join > peruser/user$f.mi
  perl scripts/calc_mi.pl < peruser/user$f.join2 > peruser/user$f.mi2

  python scripts/tf-idf.py peruser/user$f.join $totalusers gen/word2nuser.txt > peruser/user$f.tfidf
  python scripts/tf-idf.py peruser/user$f.join2 $totalusers gen/word2nuser.txt > peruser/user$f.tfidf2

#  python x2.py peruser/user$f.join > peruser/user$f.x2
#  python x2.py peruser/user$f.join2 > peruser/user$f.x22

  # use lemmata
  perl scripts/calc_mi.pl < peruser/user$f.ljoin > peruser/user$f.lmi
  perl scripts/calc_mi.pl < peruser/user$f.ljoin2 > peruser/user$f.lmi2

  python scripts/tf-idf.py peruser/user$f.ljoin $totalusers gen/word2nuser.ltxt > peruser/user$f.ltfidf
  python scripts/tf-idf.py peruser/user$f.ljoin2 $totalusers gen/word2nuser.ltxt > peruser/user$f.ltfidf2

#  python x2.py peruser/user$f.ljoin > peruser/user$f.lx2
#  python x2.py peruser/user$f.ljoin2 > peruser/user$f.lx22

done

export LC_ALL=""
